package com.ustadzmenjawab.crawler;

import java.io.IOException;
import java.util.Set;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.google.gwt.dev.util.collect.HashSet;
import com.ustadzmenjawab.shared.Article;

public class SyariahOnlineCrawler extends CrawlerRoot {
	public static final String HOME_URL = "http://www.syariahonline.com/v2/";
	public static final String PATTERN = "syariahonline.com";
	public static void main (String[] args){
		try {
			SyariahOnlineCrawler cr = new SyariahOnlineCrawler();

			Document doc = Jsoup.connect(HOME_URL).timeout(300*1000).get();
			Elements unvisited = doc.select("a[href]");
			Set<String> visited = new HashSet<String>();
			int count_relevant = 0;
			while(unvisited.size() > 0){
				Element link = unvisited.first();
			//	for(Element link:  articlesLs){
					try{
						String absHref = link.attr("abs:href");
						unvisited.remove(link);
						if(!visited.contains(absHref.toLowerCase())){
							visited.add(absHref.toLowerCase());
//							if(count_relevant == 50)
//								return;
							if(cr.isRelevantURL(absHref)){
								count_relevant++;
								Document page = Jsoup.connect(absHref).timeout(0).get();
								String content = page.html();
								Elements pageLinks = page.select("a[href]");
								for(Element newlink : pageLinks){
									String newLinkStr = newlink.attr("abs:href").toLowerCase();
									if(!visited.contains(newLinkStr) && cr.isRelevantURL(newLinkStr))
									{
										unvisited.add(newlink);
									}
								}
								if(cr.isRelevantPage(content)){
									System.out.println(absHref + " is a relevant page");
									
									if(!cr.isUrlExist(absHref)){
										cr.insertContent(absHref, content);
									}
								}else{
									System.out.println(absHref +" is not a relevant page");
								}
							}
						}
					}catch(Exception e){
						System.out.println(" exception "+e.getClass() +" :" + e.getMessage());
						e.printStackTrace();
					}
				}


			//}
		}
		catch (IOException e) {
			e.printStackTrace();
		}

	}

	@Override
	public boolean isRelevantURL(String url) {
		if(url.contains("/tentang-kami")
				||url.contains("/links")
				||url.contains("/registrasi")
				||url.contains("/konsultasi-syariah")
				||url.contains("/tadabur-quran")
				||url.contains("/mutiara-hadist")
				||url.contains("/bayan")
				||url.contains("format=pdf")
				||url.contains("format=feed")
				||url.contains("addthis.com")
				||url.contains("mailto")
				||url.contains("print=1")
				||url.contains("jigsaw.w3")
				||url.contains("/fatwa")
				||url.contains("/kirim-pertanyaan")
				||url.contains("/donatur")
				||url.contains("/pertanyaan")
				||url.contains("http://www.syariahonline.com/v2/index.php")
				||url.contains("/component/content/")
				||url.endsWith(".pdf")
				||url.equalsIgnoreCase("http://www.syariahonline.com/v2/")
		){
			return false;
		}
		if(url.contains(PATTERN))
			return true;
		return false;
	}

	@Override
	public boolean isRelevantPage(String content) {
		if(content.contains("Terbaru Lainnya") || content.contains("More Articles...") 
		){
			return false;
		}
		if(content.contains("Next ") || content.contains("Prev ")){
			return true;
		}
		return false;
	}

	@Override
	public String getTitle(String body) {
		Document doc = Jsoup.parse(body);
		Element p = doc.select("title").first();
		return p.text();
	}

	@Override
	public String getQuestion(String body) {
		int istart = body.indexOf("Pertanyaan:");
		if(istart == -1) 
			istart =body.indexOf("pertanyaan:");
		if(istart == -1)
			istart = body.indexOf("Pertanyaan");
		int iend = body.indexOf("Jawaban:");
		if(iend == -1)
			iend = body.indexOf("jawaban:");
		if(iend == -1)
			iend = body.indexOf("wassalam");
		if(iend == -1)
			iend = body.indexOf("Wassalam");

		String subbody = body.substring(istart, iend);
		Document doc = Jsoup.parse(subbody);
		String text = doc.body().text();
		return text;
	}

	@Override
	public String getArticle(String body) {
		int istart = body.indexOf("Pertanyaan:");
		if(istart == -1)
			istart = body.indexOf("Pertanyaan");
		int iend = body.indexOf("Next");
		if(iend == -1)
			iend = body.indexOf("Prev");
		String subbody = body.substring(istart, iend);
		Document doc = Jsoup.parse(subbody);
		String text = doc.body().text();
		return text;
	}

	@Override
	public boolean insertContent(String url, String content) {
		try{
			Article article = new Article();
			String title = getTitle(content);
			if(isTitleExist(title, "syariahonline.com")){
				return false;
			}
			article.setJudul(title);
			article.setKategori("shariah");
			article.setSumber("syariahonline.com");
			article.setTahun("2011");
			article.setUrl(url);
			article.setUstadz("Konsultan Syariah Online");
			article.setArtikel(getArticle(content));
			article.setPertanyaan(getQuestion(content));
			db.insertArticle(article);
			return true;
		}catch(Exception e){
			System.out.println(" exception "+e.getClass() +" :" + e.getMessage());
			e.printStackTrace();
			insertFailure(url,"exception "+e.getClass() +" :" + e.getMessage());
			return false;
		}
	}

}
